import pycantonese
import requests
#import pymongo
import bs4
from bs4 import BeautifulSoup
import time
from urllib import parse
import os
import random

# Save all data to MongoDB

# client = pymongo.MongoClient("mongodb://localhost:27017/")
# db = client['feitsui']
# singer_coll = db['singers']
# song_coll = db['lyric']

headers = {
    "Sec-Fetch-Mode": "navigate",
    "Sec-Fetch-Site": "none",
    "Sec-Fetch-User": "?1",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
    "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7"
}

agents = [
    "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/537.75.14",
    "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)",
    'Mozilla/5.0 (Windows; U; Windows NT 5.1; it; rv:1.8.1.11) Gecko/20071127 Firefox/2.0.0.11',
    'Opera/9.25 (Windows NT 5.1; U; en)',
    'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)',
    'Mozilla/5.0 (compatible; Konqueror/3.5; Linux) KHTML/3.5.5 (like Gecko) (Kubuntu)',
    'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.0.12) Gecko/20070731 Ubuntu/dapper-security Firefox/1.5.0.12',
    'Lynx/2.8.5rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/1.2.9',
    "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.7 (KHTML, like Gecko) Ubuntu/11.04 Chromium/16.0.912.77 Chrome/16.0.912.77 Safari/535.7",
    "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:10.0) Gecko/20100101 Firefox/10.0 "
]


def num_there(s):
    return any(i.isdigit() for i in s)


def download_one_song(song_url):
    try:
        headers["User-Agent"] = random.choice(agents)
        doc = requests.get(song_url, headers=headers)
        if doc.status_code != 200:
            print("[Faild] {}".format(song_url))
            return
        html = BeautifulSoup(doc.content, "lxml")
        main = html.select_one("body > div.container > div > main")
        title = main.find("h2").text.strip()

        article = main.select_one(
            'body > div.container > div > main > article').find_all('p')
        ps = [child for child in main.children if child.name == 'p']
        info = [p.split(" ") for p in ps[0].text.strip().split("\n")]
        info = {p[0]: p[1:] for p in info}
        if info.get('歌手'):
            info['歌手'] = [" ".join(info['歌手'])]
        label = []
        desc = ""
        if len(ps) == 5:
            desc = ps[1].text.strip()
            label = [p.strip() for p in ps[2].text.strip().split("\n")]
        elif len(ps) == 4:
            desc = ""
            label = [p.strip() for p in ps[1].text.strip().split("\n")]
        elif len(ps) == 3:
            desc = ""
            label = []

        lyric = []
        pinyin = []
        pinyin_notone = []
        for i, p in enumerate(article):
            for item in p.children:
                if isinstance(item, bs4.element.NavigableString):
                    item = str(item)
                    item = item.replace("(", "")
                    item = item.replace(")", "")
                    item = item.strip()
                    if "翡翠粤语歌词" not in item:
                        if not num_there(item):
                            item = item.replace(" ", "")
 #                           print(item, len(item))
                            if len(item) == 0:
                                continue
                            lyric.append(item.strip())
                        else:
                            pys = item.split()
                            pys_new_notone = []
                            pys_new = []
                            for py in pys:
                                jp = py.split('/')[0]
                                if jp[-1].isdigit() and jp[-2].isdigit():
                                    jp = jp[:-1]
                                if not jp[-1].isdigit():
#                                    print(jp)
                                    jp = pycantonese.parse_jyutping(
                                        jp+'1')[0]

                                    pys_new.append(jp.final)
                                    pys_new_notone.append(jp.final)
                                    pinyin.append(str(pys_new))
                                else:
                                    jp = pycantonese.parse_jyutping(
                                        jp)[0]

                                    pys_new.append(jp.final + jp.tone)
                                    pys_new_notone.append(jp.final)
                            pinyin.append(str(pys_new))
                            pinyin_notone.append(str(pys_new_notone))

#        print(lyric)
 #       print(pinyin)
  #      print(pinyin_notone)
        with open("cantonese_sentences.txt", "a", encoding="UTF-8") as f:
            for j in lyric:
                f.write(j+"\n")

        with open("cantonese_pinyin.txt", "a", encoding="UTF-8") as f:
            for j in pinyin:
                f.write(j+"\n")

        with open("cantonese_pinyin_notone.txt", "a", encoding="UTF-8") as f:
            for j in pinyin_notone:
                f.write(j+"\n")

        #song_coll.update_one({'url': song_url}, {"$set": doc}, upsert=True)
        #print(song_url, doc)
    except Exception as e:
        print("[Faild] URL {} {}".format(song_url, e))


song_urls = []


if __name__ == "__main__":

    count = 0
    for i in range(1, 6940):
        song_url = f'https://www.feitsui.com/zh-hans/lyrics/{i}'
        print("Downloading [{}] {}".format(i, song_url))
        download_one_song(song_url)
        time.sleep(20)  # 20的时候不会被封IP
    #print (" {} already downloa.".format(song_url))
    # """

# 961

